# # -*- coding: utf-8 -*-
# import math
# from scrapy import Request
# from scrapy.utils.project import get_project_settings
# from scrapy.exceptions import IgnoreRequest
# from zc_core.util.batch_gen import time_to_batch_no
# from zc_core.util.http_util import retry_request
# import re
# from sdszfcg.rules import *
#
#
# class SkuSpider(BaseSpider):
#     name = 'sku'
#     # 常用链接    'http://ggzyjyzx.shandong.gov.cn/wssc/mc_goods/198?type=1&page=1&rows=16&pages=276&orderby=sales&isAsc=false'
#     sku_list_url = 'http://ggzyjyzx.shandong.gov.cn/wssc/search/product?page={}&rows={}&keywords=&orderby=price&isAsc=true'
#     # 废弃三级分页接口
#     # cata_url = "http://ggzyjyzx.shandong.gov.cn/wssc/portalController/mallList?system=emall"
#     # 新三级分页接口
#     cata_url_1 = 'http://ggzyjyzx.shandong.gov.cn/wssc/portalController/mallList?type=1'
#     cata_url_2 = 'http://ggzyjyzx.shandong.gov.cn/wssc/portalController/mallList?type=2'
#     cata_url_3 = 'http://ggzyjyzx.shandong.gov.cn/wssc/portalController/mallList?type=6'
#     cata_url_4 = 'http://ggzyjyzx.shandong.gov.cn/wssc/portalController/centralList'
#     cata_url_5 = 'http://ggzyjyzx.shandong.gov.cn/wssc/portalController/fixList'
#
#     def __init__(self, batchNo=None, *args, **kwargs):
#         super(SkuSpider, self).__init__(*args, **kwargs)
#         if not batchNo:
#             self.batch_no = time_to_batch_no(datetime.now())
#         else:
#             self.batch_no = int(batchNo)
#         settings = get_project_settings()
#         self.ladders = settings.get('PRICE_LADDER', [])
#         self.rows = 150
#         self.max_page_limit = math.ceil(10000 / self.rows)
#
#     def start_requests(self):
#
#         # 采集sku第一页
#         page = 1
#         request_url = self.sku_list_url.format(page, self.rows)
#         yield Request(
#             url=request_url,
#             method='GET',
#             meta={
#                 'reqType': 'sku',
#                 'batchNo': self.batch_no
#             },
#             headers={
#                 'siteCode': 'sdszfcg'
#             },
#             callback=self.parse_sku_list,
#             errback=self.error_back
#         )
#
#     def parse_sku_list(self, response):
#         total = json.loads(response.text).get('page', {}).get('total')
#         self.logger.info('商品数量: [%s]' % total)
#
#         sku_list = parse_sku(response)
#         if sku_list:
#             self.logger.info('清单: price=<%s>, page=%s, cnt=%s' % (price_params, cur_page, len(sku_list)))
#             yield Box('sku', self.batch_no, sku_list)
#
#             # yield Box('item', self.batch_no, item_list)
#
#         # for idx in range(1, len(self.ladders)):
#         #     start_price = self.ladders[idx - 1]
#         #     # 冗余
#         #     end_price = self.ladders[idx] + 1
#         #     page = 1
#         #     price_params = "&priceRanges%5B%5D={}_{}".format(start_price, end_price)
#         #     request_url = self.sku_list_url.format(page, self.rows) + price_params
#         #     # print("url", request_url)
#         #     yield Request(
#         #         url=request_url,
#         #         method='GET',
#         #         meta={
#         #             'reqType': 'sku',
#         #             'batchNo': self.batch_no,
#         #             'priceParams': price_params,
#         #             'pageNo': page,
#         #             'flag': True
#         #         },
#         #         headers={
#         #             'Referer': 'http://ggzyjyzx.shandong.gov.cn/wssc/sdszfcg/mall/html/search.html?keywords_',
#         #             'siteCode': 'sdszfcg'
#         #         },
#         #         callback=self.parse_more_sku,
#         #         errback=self.error_back,
#         #         priority=50,
#         #         dont_filter=True
#         #     )
#
#     # def parse_more_sku(self, response):
#     #     meta = response.meta
#     #     cur_page = meta.get('pageNo')
#     #     price_params = meta.get('priceParams')
#     #     flag = meta.get('flag')
#     #     totalPage = meta.get('totalPage')
#     #     # print(cur_page, price_params, response.url)
#     #     sku_list = parse_sku(response)
#     #     if sku_list:
#     #         self.logger.info('清单: price=<%s>, page=%s, cnt=%s' % (price_params, cur_page, len(sku_list)))
#     #         yield Box('sku', self.batch_no, sku_list)
#     #         # yield Box('item', self.batch_no, item_list)
#     #
#     #         # 有更多页，并发请求
#     #         if cur_page == 1 and len(sku_list) >= self.rows:
#     #             total = json.loads(response.text)['page']['total']
#     #             total_page = math.ceil(int(total) / self.rows)
#     #             self.logger.info('总清单: price=<%s>,curPage=%s totalPage=%s' % (price_params, cur_page, total_page))
#     #             if total_page == 0 or total_page > self.max_page_limit:
#     #                 self.logger.error('页数超限: price=<%s>, total=%s' % (price_params, total_page))
#     #                 price_params_list = re.search("&priceRanges%5B%5D=(.*)", price_params).group(1).split('_')
#     #                 if int(price_params_list[1]) - int(price_params_list[0]) != 0:
#     #                     mean = int((int(price_params_list[0]) + int(price_params_list[1])) / 2)
#     #                     price_params_list = ["&priceRanges%5B%5D=" + str(price_params_list[0]) + "_" + str(mean),
#     #                                          "&priceRanges%5B%5D=" +
#     #                                          str(mean + 1) + "_" + str(price_params_list[1])]
#     #                     for price_params in price_params_list:
#     #                         request_url = self.sku_list_url.format(1, self.rows) + price_params
#     #                         yield Request(
#     #                             url=request_url,
#     #                             method='GET',
#     #                             meta={
#     #                                 'reqType': 'sku',
#     #                                 'batchNo': self.batch_no,
#     #                                 'priceParams': price_params,
#     #                                 'pageNo': 1,
#     #                                 'totalPage': total_page,
#     #                             },
#     #                             headers={
#     #                                 'Referer': 'http://ggzyjyzx.shandong.gov.cn/wssc/sdszfcg/mall/html/search.html?keywords_',
#     #                                 'siteCode': 'sdszfcg'
#     #                             },
#     #                             callback=self.parse_more_sku,
#     #                             errback=self.error_back,
#     #                             priority=50,
#     #                             dont_filter=True
#     #                         )
#     #                 else:
#     #                     self.logger.info(
#     #                         '维度：价格：状态：异常【无法进行分页】 价格区间: price=<%s>, curPage=%s,totalPage=%s' % (
#     #                             price_params, cur_page, total_page))
#     #             else:
#     #                 for idx in range(2, total_page + 1):
#     #                     request_url = self.sku_list_url.format(idx, self.rows) + price_params
#     #                     yield Request(
#     #                         url=request_url,
#     #                         method='GET',
#     #                         meta={
#     #                             'reqType': 'sku',
#     #                             'batchNo': self.batch_no,
#     #                             'priceParams': price_params,
#     #                             'pageNo': idx,
#     #                             'totalPage': total_page,
#     #                         },
#     #                         headers={
#     #                             'Referer': 'http://ggzyjyzx.shandong.gov.cn/wssc/sdszfcg/mall/html/search.html?keywords_',
#     #                             'siteCode': 'sdszfcg'
#     #                         },
#     #                         callback=self.parse_more_sku,
#     #                         errback=self.error_back,
#     #                         priority=50,
#     #                         dont_filter=True
#     #                     )
#     #     else:
#     #         self.logger.info('无商品:  price=<%s>, page=%s' % (price_params, cur_page))
#
#     def error_back(self, e):
#         if e.type and e.type == IgnoreRequest:
#             self.logger.info(e.value)
#         else:
#             if e.request:
#                 self.logger.error('请求异常: [%s][%s] -> [%s]' % (str(type(e)), e.request.url, e.request.meta))
#                 yield retry_request(e.request)
#             else:
#                 self.logger.error('未知异常: %s' % e)
